{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyP9rGid9c/02pK6fB4pV9NS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sugarforever/LlamaIndex-Tutorials/blob/main/05_Documents_Nodes/05_Documents_Nodes.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Documents & Nodes"
      ],
      "metadata": {
        "id": "-VjYJcfAkTTu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "3gyCNIM1Im7-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e047ccf8-896a-4abc-c373-0ab3a564a7e0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m868.1/868.1 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m66.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m77.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.0/77.0 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.4/143.4 kB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.0/40.0 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install -q -U llama-index"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Construct Document"
      ],
      "metadata": {
        "id": "-gRIz-JDkWNb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index import Document\n",
        "text_list = [\"hello\", \"world\"]\n",
        "documents = [Document(text=t) for t in text_list]"
      ],
      "metadata": {
        "id": "HlCpu-fhSB32"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "documents"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5xjbm28ESRHB",
        "outputId": "de5fa42c-cc34-4851-e191-1c414bdeef25"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Document(id_='7335bb8a-56dc-4074-8e79-0406237cadb1', embedding=None, metadata={}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='7debe8d278fe6c55c45f979269ab268102d75f8d48644d244cd0050dae0846ac', text='hello', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
              " Document(id_='30041a91-a8e4-448f-971a-8203c0c77e06', embedding=None, metadata={}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='12d52a924ac6bc76ec4101c5d1f55bb5b5365d5f4c524a21d6175a4b049a1962', text='world', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')]"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Customize Document"
      ],
      "metadata": {
        "id": "RSFhimunSvIw"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Metadata customization"
      ],
      "metadata": {
        "id": "va6j1fZ1WCuN"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "1. Customize in document construction"
      ],
      "metadata": {
        "id": "ARSXGkrjS1GB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index import Document\n",
        "document = Document(\n",
        "  text='Hello World',\n",
        "  metadata={\n",
        "    'filename': 'hello_world.pdf',\n",
        "    'category': 'science'\n",
        "  }\n",
        ")\n",
        "document"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WXOi9jtbTECK",
        "outputId": "6028214f-2fd3-4ce3-8f73-6f06d0b6505f"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Document(id_='e38ed14a-b41b-439e-858a-10ac815b95bb', embedding=None, metadata={'filename': 'hello_world.pdf', 'category': 'science'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='5b1ac0318e8c80ae4c7d187cd698067606fd001256065952e78d7f6229d62e26', text='Hello World', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "2. Customize after document is constructed"
      ],
      "metadata": {
        "id": "jnvmC0sMS56a"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "document.metadata = {'filename': 'hello_world_v2.pdf'}\n",
        "document"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i4TMSsKUTMTF",
        "outputId": "6179448e-cb12-4049-a1bd-3b8b5ddd727b"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Document(id_='e38ed14a-b41b-439e-858a-10ac815b95bb', embedding=None, metadata={'filename': 'hello_world_v2.pdf'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='5b1ac0318e8c80ae4c7d187cd698067606fd001256065952e78d7f6229d62e26', text='Hello World', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "3. Customize in `SimpleDirectoryReader` usage"
      ],
      "metadata": {
        "id": "s72NT50tS-Vb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf data && mkdir data\n",
        "!echo 'hello llama!' > data/hello_llama.txt\n",
        "\n",
        "from llama_index import SimpleDirectoryReader\n",
        "filenama_hook = lambda filename: {'file_name': filename}\n",
        "documents = SimpleDirectoryReader('./data', file_metadata=filenama_hook).load_data()\n",
        "documents"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bqyplmYZTSI5",
        "outputId": "a34d5bab-3eb2-4c1b-e86d-d284acf58446"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Document(id_='15db2314-2da5-4221-bd71-54c2e66b4908', embedding=None, metadata={'file_name': 'data/hello_llama.txt'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='9787e1e664b99bd6bd78fbcc816ab052d8f0808e5fcda36a9d89d360ede017b6', text='hello llama!\\n', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Document ID Customization"
      ],
      "metadata": {
        "id": "BJ71E5AsWFha"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index import Document\n",
        "document = Document(text='Hello World')\n",
        "document.doc_id = \"xxxx-yyyy\"\n",
        "document"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "btT5kT-CWIQd",
        "outputId": "39f3ef88-edda-4cd9-9ba0-ca616a0661d5"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Document(id_='xxxx-yyyy', embedding=None, metadata={}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='a339d80c761418702b627851cbd963eb722f12dfeb360d441f9123c2d3d4fcf7', text='Hello World', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Construct Node\n",
        "\n",
        "Nodes are a first-class citizen in LlamaIndex. Developers can choose to define Nodes and all its attributes directly or parse source Documents into Nodes through the `NodeParser` classes."
      ],
      "metadata": {
        "id": "BKTqmxHJTw0e"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "1. Construct directly"
      ],
      "metadata": {
        "id": "YJlU3R-0UGfp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.schema import TextNode\n",
        "node = TextNode(text=\"hello world\", id_=\"1234-5678\")\n",
        "node"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wKesz2hsUMae",
        "outputId": "04a23337-7e1d-4b81-8785-10331ee14728"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TextNode(id_='1234-5678', embedding=None, metadata={}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='351b5b79b4cf0d96b24050f568933500d93160ec03e5e996aa31c29cf2d2f654', text='hello world', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "2. Construct by NodeParser\n",
        "\n",
        "  We will reuse the `documents` variable loaded above."
      ],
      "metadata": {
        "id": "GoF5FZmyUKt3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.node_parser import SimpleNodeParser\n",
        "\n",
        "parser = SimpleNodeParser.from_defaults()\n",
        "nodes = parser.get_nodes_from_documents(documents)\n",
        "nodes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wz6SWDU6UUs7",
        "outputId": "4112e9fd-b67e-4bcb-c159-439b29d4c988"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /tmp/llama_index...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[TextNode(id_='45d7982c-c9d1-4384-b0e0-f32defff4722', embedding=None, metadata={'file_name': 'data/hello_llama.txt'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='15db2314-2da5-4221-bd71-54c2e66b4908', node_type=None, metadata={'file_name': 'data/hello_llama.txt'}, hash='9787e1e664b99bd6bd78fbcc816ab052d8f0808e5fcda36a9d89d360ede017b6')}, hash='9d29d9fb54e3e9054a39b57a34665527f97c3bf22b6f3cd2b7fdc1d72d440527', text='hello llama!', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Customize Node\n"
      ],
      "metadata": {
        "id": "rHi-taOrUsQN"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "1. Define nodes relationships"
      ],
      "metadata": {
        "id": "tMF9aWZ2UxMZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.schema import TextNode, NodeRelationship, RelatedNodeInfo\n",
        "\n",
        "hello_node = TextNode(text=\"Hello\", id_=\"1111-1111\")\n",
        "world_node = TextNode(text=\"World\", id_=\"2222-2222\")\n",
        "\n",
        "hello_node.relationships[NodeRelationship.NEXT] = RelatedNodeInfo(node_id=world_node.node_id, metadata={\"created_by\": \"VerySmallWoods\"})\n",
        "world_node.relationships[NodeRelationship.PREVIOUS] = RelatedNodeInfo(node_id=hello_node.node_id)\n",
        "nodes = [hello_node, world_node]\n",
        "nodes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N4iTvVf6Uu1N",
        "outputId": "67c5a97e-2241-4e14-fad5-6fdbb44ed854"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[TextNode(id_='1111-1111', embedding=None, metadata={}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='2222-2222', node_type=None, metadata={'created_by': 'VerySmallWoods'}, hash=None)}, hash='378b8aef7d6589c0b81c83cb8eaa637088d5ae46b9c27fd5ce9d4c56ed676b57', text='Hello', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'),\n",
              " TextNode(id_='2222-2222', embedding=None, metadata={}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='1111-1111', node_type=None, metadata={}, hash=None)}, hash='30cc2bcfbe4881f1655525a2a613acae3ab9cc0800b5bf21cac89fce84e8197e', text='World', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')]"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "2. Customize node ID"
      ],
      "metadata": {
        "id": "pur6GMrKU1v2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.schema import TextNode, NodeRelationship, RelatedNodeInfo\n",
        "\n",
        "hello_node = TextNode(text=\"Hello\", id_=\"1111-1111\")\n",
        "hello_node.id_ = '3333-3333'\n",
        "hello_node"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i6cDdDuXU4-3",
        "outputId": "4f2f3311-cfd5-4eff-faa7-a1bebe0738b4"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TextNode(id_='xx', embedding=None, metadata={}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='378b8aef7d6589c0b81c83cb8eaa637088d5ae46b9c27fd5ce9d4c56ed676b57', text='Hello', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    }
  ]
}